package cn.cas.mango.service;

import cn.cas.mango.dto.param.AggParam;
import cn.cas.mango.dto.param.News;
import cn.cas.mango.dto.param.NewsCluster;
import cn.cas.mango.util.CheckParam;
import cn.cas.mango.util.Pair;
import cn.cas.mango.util.StringUtil;
import cn.cas.mango.util.TimeUtil;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.aggregations.Aggregate;
import co.elastic.clients.elasticsearch._types.aggregations.Buckets;
import co.elastic.clients.elasticsearch._types.aggregations.StringTermsAggregate;
import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket;
import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import co.elastic.clients.elasticsearch.core.search.Hit;

import co.elastic.clients.util.ObjectBuilder;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.opencsv.CSVWriter;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.FileInputStream;

import java.io.FileWriter;
import java.io.IOException;
import java.net.URLEncoder;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;

@Service
@Slf4j
public class GetClusterInfo {
  @Autowired
  private ElasticsearchClient client;

  @Autowired
  private RequestService requestService;

  @Autowired
  private PrintJson printJson;

  @Value("${elasticsearch.newIndex}")
  private String INDEX;

  @Value("${elasticsearch.clusterIndex}")
  private String CLUSTERINDEX;

  int count = 0;

  private List<Pair<String, Long>> getGroups() {
    SearchResponse<Map> search;
    try {
      search = client.search(x -> x.index(CLUSTERINDEX)
          .aggregations("cluster", y->y.terms(t->t.field("type"))), Map.class);

    } catch (Exception e) {
      throw new RuntimeException(e);
    }
    Map<String, Aggregate> aggregations = search.aggregations();
    Aggregate cluster = aggregations.get("cluster");
    StringTermsAggregate stringTermsAggregate = (StringTermsAggregate) cluster._get();
    Buckets<StringTermsBucket> buckets = stringTermsAggregate.buckets();
    List<Pair<String, Long>> groups = new ArrayList<>();
    for (StringTermsBucket bucket: (List<StringTermsBucket>)buckets._get()) {
      String type = (String)bucket.key()._get();
      long count = bucket.docCount();
      groups.add(new Pair<>(type, count));
    }
    try {
      search = client.search(x->x.index(INDEX).size(0).query(q->q.matchAll(v->v)).trackTotalHits(v->v.enabled(true)), Map.class);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    long value = search.hits().total().value();
    groups.add(new Pair<>("-1", value));
    groups.sort((x,y)->-x.getKey().compareTo(y.getKey()));
    return groups;
  }

  public JSONArray getInfo() {
    JSONArray ret = new JSONArray();
    List<Pair<String, Long>> groups = getGroups();
    for (Pair<String, Long> pair: groups) {
      String key = pair.getKey();
      Long value = pair.getValue();
      JSONObject temp = new JSONObject(true);
      temp.put("type", key);
      temp.put("count", value);
      ret.add(temp);
    }
    return ret;
  }

  public JSONArray getAll(AggParam aggParam) {
    if (aggParam.getSize() == null) aggParam.setSize(1);
    List<String> collect = new ArrayList<>();
    Map<String, Map<String, NewsCluster>> map = new LinkedHashMap<>();
    getMap(map, collect);
    JSONArray ret = new JSONArray();
    Map<String, Map<String, Integer>> allCount = processMap(map);
    for (String type: allCount.keySet()) {
      Map<String, Integer> now = allCount.get(type);
      JSONObject temp = new JSONObject();
      JSONObject jsonObject = new JSONObject();
      int sizeGt1 = now.values().stream().filter(x -> x > aggParam.getSize()).collect(Collectors.toList()).size();
      int other = now.size() - sizeGt1;
      jsonObject.put(">" + aggParam.getSize(), sizeGt1);
      jsonObject.put("<=" + aggParam.getSize(), other);
      temp.put(type, jsonObject);
      ret.add(temp);
    }
    return ret;
  }

  private Map<String, NewsCluster> getTypeDetail(String type) {
    SearchResponse<NewsCluster> search;
    SearchRequest.Builder builder1 = new SearchRequest.Builder()
        .index(CLUSTERINDEX)
        .query(q -> q.term(t -> t.field("type").value(type)));
    SearchRequest.Builder builder2 = new SearchRequest.Builder()
        .index(CLUSTERINDEX)
        .query(q -> q.term(t -> t.field("type").value(type)));
    List<NewsCluster> large = requestService.getLarge(builder1, builder2, NewsCluster.class);
    HashMap<String, NewsCluster> ret = new HashMap<>();
    large.forEach(x->{ret.put(x.getId(), x);});
    return ret;
  }



  private Map<String, List<News>> getAllNews(List<String> clusterIds, int size) {
    Map<String, List<News>> ret = new HashMap<>();
    try {
      SearchResponse<News> search = client.search(x -> x.index(INDEX).size(size).query(q -> q.terms(t -> t.field("cluster_id3").terms(v -> v.value(
          clusterIds.stream().map(FieldValue::of).collect(Collectors.toList()))))), News.class);
      List<Hit<News>> hits = search.hits().hits();
      for (Hit<News> hit: hits) {
        News news = hit.source();
        String clusterId3 = news.getCluster_id3();
        if (!ret.containsKey(clusterId3)) {
          ret.put(clusterId3, new ArrayList<>());
        }
        ret.get(clusterId3).add(news);

      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    return ret;
  }



  private List<News> getAllNews() {
    SearchRequest.Builder builder1 = new SearchRequest.Builder().index(INDEX);
    SearchRequest.Builder builder2 = new SearchRequest.Builder().index(INDEX);
    return requestService.getLarge(builder1, builder2, News.class);
  }

  private JSONArray recur(Map<String, Map<String, NewsCluster>> map,
                          Map<String, List<News>> allNews,
                          String type,
                          Set<NewsCluster> newsClusters ) {
    JSONArray ret = new JSONArray();
    if (type.equals("0")) {
      for (NewsCluster newsCluster: newsClusters) {
        String clusterId = newsCluster.getId();
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("name", StringUtil.getCutName(newsCluster.getTitle(),10));
        jsonObject.put("children", allNews.get(clusterId).stream().map
            (x->{JSONObject temp = new JSONObject();
              String cutName = StringUtil.getCutName(x.getSjmc(), 10);
              temp.put("name", cutName);
              temp.put("value", 1);
              return temp;
            }).collect(Collectors.toList()));
        ret.add(jsonObject);
      }
    } else {
      int i = Integer.parseInt(type) - 1;
      String nextType = String.valueOf(i);
      for (NewsCluster newsCluster: newsClusters) {
        JSONObject jsonObject = new JSONObject();

        List<String> children = new ArrayList<>();
        Object child = newsCluster.getChild();
//        if (child != null) {
        if (child instanceof List) {
          children.addAll((List<String>) child);
        } else {
          children.add((String) child);
        }
        Set<NewsCluster> find = new LinkedHashSet<>();
        for (String key: children) {
          find.add(map.get(nextType).get(key));
        }
        if (children.size() == 1) {
          jsonObject = (JSONObject) recur(map, allNews, nextType, find).get(0);
        } else {
          jsonObject.put("name", StringUtil.getCutName(newsCluster.getTitle(), 10));
          jsonObject.put("children", recur(map, allNews, nextType, find));
        }

//        }
//        else { // 冗余
//          JSONObject temp = new JSONObject();
//          temp.put("name", StringUtil.getCutName(newsCluster.getTitle(), 10));
//          temp.put("value", 1);
//          jsonObject.put("children", temp);
//        }
        ret.add(jsonObject);
      }
    }
    return ret;
  }



  public JSONObject getD3Cluster(AggParam aggParam) {
    CheckParam.checkAgg(aggParam);
    // size 规定大小
    JSONObject ret = new JSONObject();
    List<String> collect = getGroups().stream().map(Pair::getKey).collect(Collectors.toList());
    Map<String, Map<String, NewsCluster>> map = new LinkedHashMap<>();
    String first = null;
    for (String type: collect) {
      map.put(type, getTypeDetail(type));
      if (first == null) {
        first = type;
      }
    }
    Map<String, NewsCluster> value = map.get("0");
    Map<String, List<News>> allNews = getAllNews(value.values().stream()
        .map(NewsCluster::getId).collect(Collectors.toList()), aggParam.getSize());

    // 拼接多个组和新闻
    ret.put("name", "flare");
    ret.put("children", recur(map, allNews, first, new LinkedHashSet<>(map.get(first).values())));
    return ret;
  }


  private JSONObject wrapOriginalNews(News x) {
    JSONObject temp = new JSONObject();
    temp.put("layer", "0");
    temp.put("id", x.getNm());
    temp.put("nm", x.getNm());
    temp.put("fssj", Strings.isBlank(x.getFssj()) ? "2010-01-01": x.getFssj());
    temp.put("sjms", Strings.isBlank(x.getSjms()) ? "": x.getSjms());
//    temp.put("keywords", x.getFilteredKeywords(printJson.getBlackListSet()));
              temp.put("lyss", x.getLyss());
    temp.put("sjmc", Strings.isBlank(x.getSjmc()) ? "": x.getSjmc());
              temp.put("ssly", x.getSsly());
    return temp;
  }

  class Recur{
    int count = 0;
    boolean getModelOrNot;
    String summaryModel;
    public Recur(boolean getModelOrNot, String summaryModel) {
      this.getModelOrNot = getModelOrNot;
      this.summaryModel = summaryModel;
    }
    private JSONArray recur2(Map<String, Map<String, NewsCluster>> map,
                             Map<String, List<News>> allNews,
                             String type,
                             Set<NewsCluster> newsClusters ) {
      JSONArray ret = new JSONArray();
      if (type.equals("0")) {
        for (NewsCluster newsCluster: newsClusters) {
          String clusterId = newsCluster.getId();
          JSONObject jsonObject = new JSONObject();
          List<News> cluster = allNews.get(clusterId);
          if (getModelOrNot) {
            if (allNews.get(clusterId).size() == 1) {
              jsonObject.put("summary", cluster.get(0).borrowWrappedContent());
            } else {
              if (summaryModel.equals("xgs"))
                jsonObject.put("summary", requestService.get(allNews.get(clusterId).stream().map(News::borrowWrappedTitle).collect(Collectors.toList()),
                  allNews.get(clusterId).stream().map(News::borrowWrappedContent).collect(Collectors.toList()),
                  allNews.get(clusterId).stream().map(News::getFssj).collect(Collectors.toList())));
              else
                jsonObject.put("summary", requestService.get(allNews.get(clusterId).stream().map(News::borrowWrappedContent).collect(Collectors.toList())));
            }
          } else {
            jsonObject.put("summary", String.join(" ", ((List<Map>)map.get(type).get(clusterId).getKeywords()).stream().map(x->(String)x.get("name")).collect(Collectors.toList())));
          }
          jsonObject.put("layer", "1");
          jsonObject.put("id", count++);
          jsonObject.put("children", cluster.stream().map
              (x-> wrapOriginalNews(x)).collect(Collectors.toList()));
          ret.add(jsonObject);
        }
      } else {
        int i = Integer.parseInt(type) - 1;
        String nextType = String.valueOf(i);
        for (NewsCluster newsCluster: newsClusters) {
          JSONObject jsonObject = new JSONObject();

          List<String> children = new ArrayList<>();
          Object child = newsCluster.getChild();
//        if (child != null) {
          if (child instanceof List) {
            children.addAll((List<String>) child);
          } else {
            children.add((String) child);
          }
          Set<NewsCluster> find = new LinkedHashSet<>();
          for (String key: children) {
            find.add(map.get(nextType).get(key));
          }
          if (children.size() == 1 ) {
            jsonObject = (JSONObject) recur2(map, allNews, nextType, find).get(0);
          } else {
            jsonObject.put("children", recur2(map, allNews, nextType, find));
            JSONArray childrenArray = (JSONArray) jsonObject.get("children");

            List<String> summary = childrenArray.stream().map(x -> (String) ((JSONObject) x).get("summary")).collect(Collectors.toList());
            if (getModelOrNot) {
              if (summary.size() == 1) {
                jsonObject.put("summary", summary.get(0));
              } else {
                if (summaryModel.equals("xgs")) {
                  jsonObject.put("summary", requestService.get(new ArrayList<>(), summary, new ArrayList<>()));
                } else {
                  jsonObject.put("summary", requestService.get(summary));
                }

              }
            } else {
              jsonObject.put("summary", String.join(" ",
                  ((List<Map>)map.get(type).get(newsCluster.getId()).getKeywords()).stream().map(x->(String)x.get("name")).collect(Collectors.toList())));
            }

            jsonObject.put("layer", String.valueOf(Integer.parseInt(type) + 1));
            jsonObject.put("id", count++);
          }


          ret.add(jsonObject);
        }
      }
      return ret;
    }

    private JSONArray recur3(Map<String, Map<String, NewsCluster>> map,
                             Map<String, List<News>> allNews,
                             String type,
                             Set<NewsCluster> newsClusters) {
      JSONArray ret = new JSONArray();
      if (type.equals("0")) {
        for (NewsCluster newsCluster: newsClusters) {
          String clusterId = newsCluster.getId();
          JSONObject jsonObject = new JSONObject();
          List<News> cluster = allNews.get(clusterId);
          if (getModelOrNot) {
            if (allNews.get(clusterId).size() == 1) {
              jsonObject.put("summary", cluster.get(0).borrowWrappedContent());
            } else {
              jsonObject.put("summary", requestService.get(allNews.get(clusterId).stream().map(News::borrowWrappedContent).collect(Collectors.toList())));
            }
          } else {
            jsonObject.put("summary", String.join(" ", ((List<Map>)map.get(type).get(clusterId).getKeywords()).stream().map(x->(String)x.get("name")).collect(Collectors.toList())));
          }

          jsonObject.put("layer", "1");
          jsonObject.put("id", count++);
          jsonObject.put("children", cluster.stream().map
              (x->wrapOriginalNews(x)).collect(Collectors.toList()));
          ret.add(jsonObject);
        }
      } else {
        int i = Integer.parseInt(type) - 1;
        String nextType = String.valueOf(i);
        for (NewsCluster newsCluster: newsClusters) {
          JSONObject jsonObject = new JSONObject();

          List<String> children = new ArrayList<>();
          Object child = newsCluster.getChild();
//        if (child != null) {
          if (child instanceof List) {
            children.addAll((List<String>) child);
          } else {
            children.add((String) child);
          }
          Set<NewsCluster> find = new LinkedHashSet<>();
          for (String key: children) {
            find.add(map.get(nextType).get(key));
          }

          jsonObject.put("children", recur3(map, allNews, nextType, find));
          JSONArray childrenArray = (JSONArray) jsonObject.get("children");

          List<String> summary = childrenArray.stream().map(x -> (String) ((JSONObject) x).get("summary")).collect(Collectors.toList());
          if (getModelOrNot) {
            if (summary.size() == 1) {
              jsonObject.put("summary", summary.get(0));
            } else {
              jsonObject.put("summary", requestService.get(summary));
            }
          } else {
            jsonObject.put("summary", String.join(" ",
                ((List<Map>)map.get(type).get(newsCluster.getId()).getKeywords()).stream().map(x->(String)x.get("name")).collect(Collectors.toList())));
          }
          jsonObject.put("layer", String.valueOf(Integer.parseInt(type) + 1));
          jsonObject.put("id", count++);
          ret.add(jsonObject);
        }
      }
      return ret;
    }
  }

//  private JSONArray recur2(Map<String, Map<String, NewsCluster>> map,
//                          Map<String, List<News>> allNews,
//                          String type,
//                          Set<NewsCluster> newsClusters ) {
//    JSONArray ret = new JSONArray();
//    if (type.equals("0")) {
//      for (NewsCluster newsCluster: newsClusters) {
//        String clusterId = newsCluster.getId();
//        JSONObject jsonObject = new JSONObject();
//        List<News> cluster = allNews.get(clusterId);
//        if (allNews.get(clusterId).size() == 1) {
//          jsonObject.put("summary", cluster.get(0).borrowWrappedTitle());
//        } else {
//          jsonObject.put("summary", requestService.get(allNews.get(clusterId).stream().map(News::borrowWrappedTitle).collect(Collectors.toList())));
//        }
//        jsonObject.put("layer", "1");
//        jsonObject.put("id", count++);
//        jsonObject.put("children", cluster.stream().map
//            (x->{JSONObject temp = new JSONObject();
//              temp.put("layer", "0");
//              temp.put("id", x.getNm());
//              temp.put("nm", x.getNm());
//              temp.put("fssj", x.getFssj());
//              temp.put("sjms", x.getSjms());
////              temp.put("lyss", x.getLyss());
//              temp.put("sjmc", x.getSjmc());
////              temp.put("ssly", x.getSsly());
//              return temp;
//            }).collect(Collectors.toList()));
//        ret.add(jsonObject);
//      }
//    } else {
//      int i = Integer.parseInt(type) - 1;
//      String nextType = String.valueOf(i);
//      for (NewsCluster newsCluster: newsClusters) {
//        JSONObject jsonObject = new JSONObject();
//
//        List<String> children = new ArrayList<>();
//        Object child = newsCluster.getChild();
////        if (child != null) {
//        if (child instanceof List) {
//          children.addAll((List<String>) child);
//        } else {
//          children.add((String) child);
//        }
//        Set<NewsCluster> find = new LinkedHashSet<>();
//        for (String key: children) {
//          find.add(map.get(nextType).get(key));
//        }
//        if (children.size() == 1 ) {
//          jsonObject = (JSONObject) recur2(map, allNews, nextType, find).get(0);
//        } else {
//          jsonObject.put("children", recur2(map, allNews, nextType, find));
//          JSONArray childrenArray = (JSONArray) jsonObject.get("children");
//
//          List<String> summary = childrenArray.stream().map(x -> (String) ((JSONObject) x).get("summary")).collect(Collectors.toList());
//          if (summary.size() == 1) {
//            jsonObject.put("summary", summary.get(0));
//          } else {
//            jsonObject.put("summary", requestService.get(summary));
//          }
//          jsonObject.put("layer", String.valueOf(Integer.parseInt(type) + 1));
//          jsonObject.put("id", count++);
//        }
//
//
//        ret.add(jsonObject);
//      }
//    }
//    return ret;
//  }
//
//  private JSONArray recur3(Map<String, Map<String, NewsCluster>> map,
//                           Map<String, List<News>> allNews,
//                           String type,
//                           Set<NewsCluster> newsClusters) {
//    JSONArray ret = new JSONArray();
//    if (type.equals("0")) {
//      for (NewsCluster newsCluster: newsClusters) {
//        String clusterId = newsCluster.getId();
//        JSONObject jsonObject = new JSONObject();
//        List<News> cluster = allNews.get(clusterId);
//        if (allNews.get(clusterId).size() == 1) {
//          jsonObject.put("summary", cluster.get(0).borrowWrappedTitle());
//        } else {
//          jsonObject.put("summary", requestService.get(allNews.get(clusterId).stream().map(News::borrowWrappedTitle).collect(Collectors.toList())));
//        }
//        jsonObject.put("layer", "1");
//        jsonObject.put("id", count++);
//        jsonObject.put("children", cluster.stream().map
//            (x->{JSONObject temp = new JSONObject();
//              temp.put("layer", "0");
//              temp.put("id", x.getNm());
//              temp.put("nm", x.getNm());
//              temp.put("fssj", x.getFssj());
//              temp.put("sjms", x.getSjms());
//              temp.put("sjmc", x.getSjmc());
////              temp.put("lyss", x.getLyss());
////              temp.put("ssly", x.getSsly());
//              return temp;
//            }).collect(Collectors.toList()));
//        ret.add(jsonObject);
//      }
//    } else {
//      int i = Integer.parseInt(type) - 1;
//      String nextType = String.valueOf(i);
//      for (NewsCluster newsCluster: newsClusters) {
//        JSONObject jsonObject = new JSONObject();
//
//        List<String> children = new ArrayList<>();
//        Object child = newsCluster.getChild();
////        if (child != null) {
//        if (child instanceof List) {
//          children.addAll((List<String>) child);
//        } else {
//          children.add((String) child);
//        }
//        Set<NewsCluster> find = new LinkedHashSet<>();
//        for (String key: children) {
//          find.add(map.get(nextType).get(key));
//        }
//
//        jsonObject.put("children", recur3(map, allNews, nextType, find));
//        JSONArray childrenArray = (JSONArray) jsonObject.get("children");
//
//        List<String> summary = childrenArray.stream().map(x -> (String) ((JSONObject) x).get("summary")).collect(Collectors.toList());
//        if (summary.size() == 1) {
//          jsonObject.put("summary", summary.get(0));
//        } else {
//          jsonObject.put("summary", requestService.get(summary));
//        }
//        jsonObject.put("layer", String.valueOf(Integer.parseInt(type) + 1));
//        jsonObject.put("id", count++);
//        ret.add(jsonObject);
//      }
//    }
//    return ret;
//  }

  public void write2Json(JSONObject jsonObject) {
    try (FileWriter file = new FileWriter("data.json")) {
      // 将JSONObject写入文件
      file.write(jsonObject.toJSONString());
      file.flush();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  public JSONObject getJson()  {
    String content = null;
    try {
      content = new String(Files.readAllBytes(Paths.get("data.json")));
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    // 将字符串解析为JSONObject
    JSONObject temp = JSONObject.parseObject(content);
    JSONObject ret = new JSONObject();
    ret.put("code", 200);
    ret.put("message", "success");
    ret.put("data", temp);
    return ret;
  }

  public JSONObject getD3Cluster2(AggParam aggParam) {
    CheckParam.checkAgg(aggParam);
    log.info("get d3 cluster start...");
    TimeUtil start = new TimeUtil();

    // size 规定大小
    JSONObject ret = new JSONObject();
    List<String> collect = getGroups().stream().map(Pair::getKey).collect(Collectors.toList());
    Map<String, Map<String, NewsCluster>> map = new LinkedHashMap<>();
    String first = null;
    for (String type: collect) {
      map.put(type, getTypeDetail(type));
      if (first == null) {
        first = type;
      }
    }
    Map<String, NewsCluster> value = map.get("0");

    Map<String, List<News>> allNews = getAllNews(value.values().stream()
        .map(NewsCluster::getId).collect(Collectors.toList()), aggParam.getSize());


      assert first != null;
      ret.put("layer", String.valueOf(Integer.parseInt(first) + 2));
    // 拼接多个组和新闻
    if (aggParam.getShow() == null)
      ret.put("children", new Recur(aggParam.getGetModel(), aggParam.getSummaryModel()).recur2(map, allNews, first, new LinkedHashSet<>(map.get(first).values())));
    else {
      ret.put("children", new Recur(aggParam.getGetModel(), aggParam.getSummaryModel()).recur3(map, allNews, first, new LinkedHashSet<>(map.get(first).values())));
    }
//    requestService.getAllCost();
    count = 0;

    log.info("get d3 cluster total cost: {}s", start.getCost());
    JSONArray jsonArray = (JSONArray) ret.get("children");
    Map<JSONObject, Integer> object2Count = new HashMap<>();
    for (Object o: jsonArray) {
      JSONObject j = (JSONObject) o;
      object2Count.put(j, getLeafSize(j));
    }
    jsonArray.sort((a,b)->-object2Count.get(a).compareTo(object2Count.get(b)));
    write2Json(ret);
    return ret;
  }

  /**
   * 对查看每个jsonobject下实际有多少叶子节点
   * @param jsonObject， 含有key： children
   * @return
   */
  private int getLeafSize(JSONObject jsonObject) {
    if (!jsonObject.containsKey("children")) {
      return 1;
    }
    int count = 0;
    for (Object o: (List) jsonObject.get("children")) {
      JSONObject j = (JSONObject) o;
      count += getLeafSize(j);
    }
    return count;
  }

  private List<String> getSortedCluster(Map<String, NewsCluster> value) {
    List<Map.Entry<String, NewsCluster>> tempValue = new ArrayList<>(value.entrySet());
    tempValue.sort((x,y)->{
      List<String> xChild = (List<String>) (x.getValue().getChild());
      List<String> yChild = (List<String>) (y.getValue().getChild());
      int xSize = xChild.size();
      int ySize = yChild.size();
      if (xSize == ySize) return 0;
      else return xSize < ySize ? 1: -1;
    });
    return tempValue.stream().map(x->x.getKey()).collect(Collectors.toList());
  }

  private List<String> getSortedCluster(Map<String, NewsCluster> value, Map<String, Integer> count) {
    List<Map.Entry<String, NewsCluster>> tempValue = new ArrayList<>(value.entrySet());
    tempValue.sort((x,y)-> -count.get(x.getKey()).compareTo(count.get(y.getKey())));
    return tempValue.stream().map(x->x.getKey()).collect(Collectors.toList());
  }

  private List<List<String>> processPos(List<List<String>> res,
                                        Map<String, Map<String, NewsCluster>> map,
                                        int max) {

    Map<String, Map<String, Integer>> allCount = processMap(map);
    for (int i = 1; i <= max; i++) {
      List<List<String>> temp = new ArrayList<>();
      String nowString = String.valueOf(i);
      Map<String, NewsCluster> stringNewsClusterMap = map.get(nowString);
      List<String> sortedCluster = getSortedCluster(stringNewsClusterMap, allCount.get(nowString));
      int index = 2 * i + 4;
      Map<String, List<List<String>>> tempMap = new LinkedHashMap<>();
      for (List<String> list: res) {
        String s = list.get(index);
        if (!tempMap.containsKey(s)) tempMap.put(s, new ArrayList<>());
        tempMap.get(s).add(list);
      }
      for (String s: sortedCluster) {
        temp.addAll(tempMap.get(s));
      }
      res = new ArrayList<>(temp);
//      for (String id: stringNewsClusterMap.keySet()) {
//        NewsCluster newsCluster = stringNewsClusterMap.get(id);
//        List<String> child = (List<String>) newsCluster.getChild();
//
//      }
    }
    return res;
  }

  public void getCsv(AggParam aggParam) {
    String folderPath = "csv";
    File folder = new File(folderPath);
    if (!folder.exists()) {
      folder.mkdir();
    }
    CheckParam.checkAgg(aggParam);
    List<String> collect = getGroups().stream().map(Pair::getKey).collect(Collectors.toList());
    Map<String, Map<String, NewsCluster>> map = new LinkedHashMap<>();
    String first = null;
    for (String type: collect) {
      map.put(type, getTypeDetail(type));
      if (first == null) {
        first = type;
      }
    }
    Map<String, NewsCluster> value = map.get("0");
    List<News> allNews = getAllNews();
    Map<String, News> allNewsMap = new HashMap<>();
    allNews.forEach(x->allNewsMap.put(x.getNm(), x));
    String fileName = "-1_0.csv";
    List<String> title = new ArrayList<>();
    title.add("content"); title.add("id"); title.add("keywords"); title.add("pId"); title.add("pKeywords");
    List<List<String>> res = new ArrayList<>();

    for (String clusterId: getSortedCluster(value)) {
      NewsCluster newsCluster = value.get(clusterId);
      List<String> child = (List<String>)newsCluster.getChild();
      for (String string: child) {
        List<String> temp = new ArrayList<>();
        News news = allNewsMap.get(string);
        temp.add(news.getSjms());
        temp.add(news.getNm());
        temp.add(news.getFilteredKeywords(printJson.getBlackListSet()).stream().reduce((x,y)->x+" " + y).orElse(""));
        temp.add(clusterId);
        temp.add(map.get("0").get(clusterId).getFilteredKeywords().stream().reduce((x,y)->x+" " + y).orElse(""));
        res.add(temp);
      }
    }
    try {
      writeCSVFile(folderPath+"/"+fileName, new ArrayList<String[]>(){{add(title.toArray(new String[0]));}},
          res.stream().map(x->x.toArray(new String[0])).collect(Collectors.toList()), ",");
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    log.info("finish -1_0.csv writing");

    collect.remove("-1");
    title.remove("content");
    int times = collect.size() - 1;

    for (int i = 1; i <= times; i++) {
      String typeParent = String.valueOf(i);
      String typeSon = String.valueOf(i - 1);
      Map<String, NewsCluster> parentMap = map.get(typeParent);
      Map<String, NewsCluster> sonMap = map.get(typeSon);
      res.clear();
      fileName = typeSon + "_" + typeParent + ".csv";
      for (String clusterId: getSortedCluster(parentMap)) {
        NewsCluster parent = parentMap.get(clusterId);
        List<String> child = (List<String>)parent.getChild();
        for (String string: child) {
          List<String> temp = new ArrayList<>();
          NewsCluster son = sonMap.get(string);
          temp.add(son.getId());
          temp.add(son.getKeywordsString());

          temp.add(clusterId);
          temp.add(parent.getKeywordsString());
          res.add(temp);
        }
      }
      try {
        writeCSVFile(folderPath+"/"+ fileName, new ArrayList<String[]>(){{add(title.toArray(new String[0]));}},
            res.stream().map(x->x.toArray(new String[0])).collect(Collectors.toList()), ",");
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      log.info("finish {}_{}.csv writing", typeSon, typeParent);
    }
  }

  private Map<String, Map<String, Integer>> processMap(Map<String, Map<String, NewsCluster>> map1) {
    Map<String, Map<String, Integer>> ret = new HashMap<>();
    Map<String, Map<String, NewsCluster>> map = new HashMap<>(map1);
    map.remove("-1");
    ArrayList<String> strings = new ArrayList<>(map.keySet());
    strings.sort((x,y)->Integer.valueOf(x).compareTo(Integer.valueOf(y)));
    for (String type: strings) {
      ret.put(type, new HashMap<>());
      if (type.equals("0")) {
        List<Integer> collect = map.get(type).values().stream().map(x -> ((List<String>) x.getChild()).size()).collect(Collectors.toList());
        int i = 0;
        for (String key: map.get(type).keySet()) {
          ret.get(type).put(key, collect.get(i++));
        }
      } else {
        String sonType = String.valueOf(Integer.parseInt(type) - 1);
        for (String key: map.get(type).keySet()) {
          List<String> child = (List<String>) (map.get(type).get(key).getChild());
          Integer sum = child.stream().map(x -> ret.get(sonType).get(x)).reduce(Integer::sum).get();
          ret.get(type).put(key, sum);
        }
      }
    }
    return ret;
  }

  private void getMap(Map<String, Map<String, NewsCluster>> map, List<String> collect) {
    collect.addAll(getGroups().stream().map(Pair::getKey).collect(Collectors.toList()));
    String first = null;
    for (String type: collect) {
      map.put(type, getTypeDetail(type));
      if (first == null) {
        first = type;
      }
    }
  }

  public void getCsv2(AggParam aggParam) {
    String folderPath = "csv";
    File folder = new File(folderPath);
    if (!folder.exists()) {
      folder.mkdir();
    }
    CheckParam.checkAgg(aggParam);
    List<String> collect = new ArrayList<>();
    Map<String, Map<String, NewsCluster>> map = new LinkedHashMap<>();
    getMap(map, collect);
    Map<String, NewsCluster> value = map.get("0");
    List<News> allNews = getAllNews();
    Map<String, News> allNewsMap = new HashMap<>();
    allNews.forEach(x->allNewsMap.put(x.getNm(), x));
    String fileName = "all.csv";
    List<String> title = new ArrayList<>();
    title.add("title");title.add("content");  title.add("id");title.add("keywords");
    collect.remove("-1");

    for (int i = 0; i < collect.size(); i++) {
      title.add(String.format("p%dId", i + 1));
      title.add(String.format("p%dKeywords", i+1));
    }

    int times = collect.size() - 1;
    List<List<String>> res = new ArrayList<>();
    Map<String, Map<String, String>> mapParent = new HashMap<>();
    for (int i = 1; i <= times; i++) {
      String typeParent = String.valueOf(i);
      String typeSon = String.valueOf(i - 1);
      if (!mapParent.containsKey(typeSon)) {
        mapParent.put(typeSon, new HashMap<>());
      }
      Map<String, String> nowTypeMap = mapParent.get(typeSon);
      Map<String, NewsCluster> parentMap = map.get(typeParent);
      for (String clusterId: getSortedCluster(parentMap)) {
        NewsCluster parent = parentMap.get(clusterId);
        List<String> child = (List<String>)parent.getChild();
        for (String string: child) {
          nowTypeMap.put(string, clusterId);
        }
      }
    }
    for (String clusterId: getSortedCluster(value)) {
      NewsCluster newsCluster = value.get(clusterId);
      List<String> child = (List<String>)newsCluster.getChild();
      List<String> add2Temp = new ArrayList<>();
      String tempId = clusterId;
      for (int j = 0; j < times; j++) {
        String sonType = String.valueOf(j);
        String parentType = String.valueOf(j + 1);
        String parentId = mapParent.get(sonType).get(tempId);
        tempId = parentId;
        String parentKeywordsString = map.get(parentType).get(parentId).getKeywordsString();
        add2Temp.add(parentId); add2Temp.add(parentKeywordsString);
      }
      for (String string: child) {
        List<String> temp = new ArrayList<>();
        News news = allNewsMap.get(string);
        temp.add(news.getSjmc());
        temp.add(news.getSjms());
        temp.add(news.getNm());
        temp.add(news.getFilteredKeywords(printJson.getBlackListSet()).stream().reduce((x,y)->x+" " + y).orElse(""));

        temp.add(clusterId);
        temp.add(map.get("0").get(clusterId).getFilteredKeywords().stream().reduce((x,y)->x+" " + y).orElse(""));
        temp.addAll(add2Temp);
        res.add(temp);
      }
    }
    res = processPos(res, map, times);
    try {
      writeCSVFile(folderPath+"/"+fileName, new ArrayList<String[]>(){{add(title.toArray(new String[0]));}},
          res.stream().map(x->x.toArray(new String[0])).collect(Collectors.toList()), ",");
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    log.info("finish all.csv writing");
    try {
      download(((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getResponse(), "csv/all.csv");
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

    private void download(HttpServletResponse response, String fileName) throws IOException {
      final Path path = Paths.get(fileName);
      System.out.println(fileName);
//        设置响应头
      response.setContentType("application/octet-stream");
      response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(fileName, "UTF-8"));
//        获取输出流
      final WritableByteChannel writableByteChannel = Channels.newChannel(response.getOutputStream());
//        读取文件流通道
      final FileChannel fileChannel = new FileInputStream(path.toFile()).getChannel();
//        写入数据到通道
      fileChannel.transferTo(0, fileChannel.size(), writableByteChannel);
      fileChannel.close();
      writableByteChannel.close();
  }


  private void writeCSVFile(String writeFileName, List<String[]> csvFileFirstRowList, List<String[]> contentArrayList, String characters) throws IOException{

    //判断文件是否存在，如果存在，先删除
    File file = new File(writeFileName);

//    if (file.exists())
//      file.delete();

    CSVWriter writer = null;
    if (",".equalsIgnoreCase(characters)){
      //初始化CSVWriter
      writer = new CSVWriter(new FileWriter(file));
    } else{
      //初始化CSVWriter，带分隔符
      writer = new CSVWriter(new FileWriter(file),characters.toCharArray()[0]);
    }
    //csv表头数据+表体数据
    csvFileFirstRowList.addAll(contentArrayList);
    writer.writeAll(csvFileFirstRowList);
    //关闭流
    writer.close();
  }




}
